import pandas as pd
import numpy as np
from sklearn.preprocessing import LabelEncoder
import pickle

# 定义批次大小
chunksize = 100000  # 根据内存情况调整批次大小

# 初始化空的列表来存储所有用户ID
all_user_ids = []
all_user_product_ids = []

# 分批次读取purchases.csv文件，提取用户ID
for chunk in pd.read_csv('purchases.csv', usecols=['user_id', 'product_id'], chunksize=chunksize):
    all_user_ids.extend(chunk['user_id'].unique())
    all_user_product_ids.extend(chunk['product_id'].unique())

# 去重
all_user_ids = np.unique(all_user_ids)
all_user_product_ids = np.unique(all_user_product_ids)

# 读取 products.csv 文件
products_df = pd.read_csv('products.csv', usecols=['product_id'])

# 确保数据类型一致
products_df['product_id'] = products_df['product_id'].astype(str).str.strip()

# 获取所有商品ID
all_product_ids = products_df['product_id'].unique()

# 输出用户ID和商品ID的数量
print(f"用户ID数量: {len(all_user_ids)}")
print(f"商品ID数量: {len(all_product_ids)}")


product_ids_set = set(all_product_ids)
purchase_ids_set = set(all_user_product_ids)
# 找出只在user_ids中存在的值
only_in_product_ids = product_ids_set - purchase_ids_set
# 找出只在product_ids中存在的值
only_in_purchase_ids = purchase_ids_set - product_ids_set
# print("只在product才中存在的值:", only_in_product_ids)
print("只在purchase中存在的值:", only_in_purchase_ids)

# 对用户和商品进行编码
le_user = LabelEncoder()
le_product = LabelEncoder()

le_user.fit(all_user_ids)
le_product.fit(all_product_ids)

# 将编码信息保存到文件
with open('user_encoder.pkl', 'wb') as f:
    pickle.dump(le_user, f)

with open('product_encoder.pkl', 'wb') as f:
    pickle.dump(le_product, f)

print("编码信息已保存。")
